In [1]:
#import library
import pandas as pd
import numpy as np
In [2]:
import matplotlib.pyplot as plt
import os
import scipy.stats as sts
%matplotlib inline
In [3]:
os.getcwd() #directory
Out[3]:
'/home/khanhp/Documents/Dev/pythonfordataanalysis'
In [4]:
ls

'Data Exploration.html'              test_predictdata.csv
'Data Exploration.ipynb'             train_predictdata.csv
 README.md                          'Transactional data.xlsx'
 sample_submission_predictdata.csv   Transactional.ipynb
In [5]:
df_train = pd.read_csv('train_predictdata.csv') #reading csv file
In [6]:
df_train.head() # first row of the dataset
Out[6]:
Loan_ID Gender Married Dependents Education Self_Employed ApplicantIncome CoapplicantIncome LoanAmount Loan_Amount_Term Credit_History Property_Area Loan_Status
0 LP001002 Male No 0 Graduate No 5849 0.0 NaN 360.0 1.0 Urban Y
1 LP001003 Male Yes 1 Graduate No 4583 1508.0 128.0 360.0 1.0 Rural N
2 LP001005 Male Yes 0 Graduate Yes 3000 0.0 66.0 360.0 1.0 Urban Y
3 LP001006 Male Yes 0 Not Graduate No 2583 2358.0 120.0 360.0 1.0 Urban Y
4 LP001008 Male No 0 Graduate No 6000 0.0 141.0 360.0 1.0 Urban Y
In [7]:
df_train.columns
Out[7]:
Index(['Loan_ID', 'Gender', 'Married', 'Dependents', 'Education',
       'Self_Employed', 'ApplicantIncome', 'CoapplicantIncome', 'LoanAmount',
       'Loan_Amount_Term', 'Credit_History', 'Property_Area', 'Loan_Status'],
      dtype='object')
Variable Description Loan_ID Unique Loan ID Gender Male/ Female Married Applicant married (Y/N) Dependents Number of dependents Education Applicant Education (Graduate/Under Graduate) Self_Employed Self employed (Y/N) ApplicantIncome Applicant income CoapplicantIncome Coapplicant income LoanAmount Loan amount in thousands Loan_Amount_Term Term of loan in months Credit_History Credit history meets guidelines Property_Area Urban/ Semi Urban/ Rural Loan_Status Loan approved (Y/N)
In [8]:
df_train.dtypes
Out[8]:
Loan_ID               object
Gender                object
Married               object
Dependents            object
Education             object
Self_Employed         object
ApplicantIncome        int64
CoapplicantIncome    float64
LoanAmount           float64
Loan_Amount_Term     float64
Credit_History       float64
Property_Area         object
Loan_Status           object
dtype: object
In [9]:
df_train.describe(include=['float64']) #get summary of numerical variables
Out[9]:
CoapplicantIncome LoanAmount Loan_Amount_Term Credit_History
count 614.000000 592.000000 600.00000 564.000000
mean 1621.245798 146.412162 342.00000 0.842199
std 2926.248369 85.587325 65.12041 0.364878
min 0.000000 9.000000 12.00000 0.000000
25% 0.000000 100.000000 360.00000 1.000000
50% 1188.500000 128.000000 360.00000 1.000000
75% 2297.250000 168.000000 360.00000 1.000000
max 41667.000000 700.000000 480.00000 1.000000
1.LoanAmount has (614 – 592) 22 missing values. 2. Loan_Amount_Term has (614 – 600) 14 missing values. 3. Credit_History has (614 – 564) 50 missing values. 4. We can also look that about 84% applicants have a credit_history. How? The mean of Credit_History field is 0.84 ( Credit_History has value 1 for those who have a credit history and 0 otherwise) 5. The ApplicantIncome distribution seems to be in line with expectation. Same with CoapplicantIncome
In [10]:
df_train['Credit_History'].value_counts()
Out[10]:
1.0    475
0.0     89
Name: Credit_History, dtype: int64
In [11]:
df_train['Loan_Status'].value_counts(normalize=True)
Out[11]:
Y    0.687296
N    0.312704
Name: Loan_Status, dtype: float64
In [12]:
df_train['Loan_Status'].value_counts().plot.bar()
Out[12]:
<matplotlib.axes._subplots.AxesSubplot at 0x7f1f21309ef0>
In [ ]:
?plt.bar
In [15]:
df_train.describe(include=['object']) #get summary of categorical variables
Out[15]:
Loan_ID Gender Married Dependents Education Self_Employed Property_Area Loan_Status
count 614 601 611 599 614 582 614 614
unique 614 2 2 4 2 2 3 2
top LP002833 Male Yes 0 Graduate No Semiurban Y
freq 1 489 398 345 480 500 233 422
In [17]:
df_train.columns
Out[17]:
Index(['Loan_ID', 'Gender', 'Married', 'Dependents', 'Education',
       'Self_Employed', 'ApplicantIncome', 'CoapplicantIncome', 'LoanAmount',
       'Loan_Amount_Term', 'Credit_History', 'Property_Area', 'Loan_Status'],
      dtype='object')
In [18]:
#Apply Function
#Create a new function:
def num_missing(x):
  return sum(x.isnull())
#Applying per column:
print("Missing values per column:")
print(df_train.apply(num_missing, axis=0)) #axis=0 defines that function is to be applied on each column

Missing values per column:
Loan_ID               0
Gender               13
Married               3
Dependents           15
Education             0
Self_Employed        32
ApplicantIncome       0
CoapplicantIncome     0
LoanAmount           22
Loan_Amount_Term     14
Credit_History       50
Property_Area         0
Loan_Status           0
dtype: int64
In [19]:
temp1 = df_train['Credit_History'].value_counts(ascending=True) 
temp2 = df_train.pivot_table(values='Loan_Status',index=['Credit_History'],aggfunc=lambda x: x.map({'Y':1,'N':0}).mean()) 
print('Frequency Table for Credit History:\n %s' %(temp1))
# print temp1 
print('\nProbility of getting loan for each Credit History class:')
print(temp2)

Frequency Table for Credit History:
 0.0     89
1.0    475
Name: Credit_History, dtype: int64

Probility of getting loan for each Credit History class:
                Loan_Status
Credit_History             
0.0                0.078652
1.0                0.795789
In [20]:
import matplotlib.pyplot as plt 
fig = plt.figure(figsize=(8,4)) 
ax1 = fig.add_subplot(121) 
ax1.set_xlabel('Credit_History') 
ax1.set_ylabel('Count of Applicants') 
ax1.set_title("Applicants by Credit_History") 
temp1.plot(kind='bar') 
ax2 = fig.add_subplot(122) 
temp2.plot(kind = 'bar') 
ax2.set_xlabel('Credit_History') 
ax2.set_ylabel('Probability of getting loan') 
ax2.set_title("Probability of getting loan by credit history")
Out[20]:
Text(0.5, 1.0, 'Probability of getting loan by credit history')
In [21]:
temp3 = pd.crosstab(df_train['Credit_History'], df_train['Loan_Status']) 
temp3.plot(kind='bar', stacked=True, color=['red','blue'], grid=False)
Out[21]:
<matplotlib.axes._subplots.AxesSubplot at 0x7f1f1ad73898>
In [22]:
table = df_train.pivot_table(values='LoanAmount', index='Self_Employed' ,columns='Education', aggfunc=np.median)
In [24]:
table
Out[24]:
Education Graduate Not Graduate
Self_Employed
No 130.0 113.0
Yes 157.5 130.0
In [23]:
df_train.columns
Out[23]:
Index(['Loan_ID', 'Gender', 'Married', 'Dependents', 'Education',
       'Self_Employed', 'ApplicantIncome', 'CoapplicantIncome', 'LoanAmount',
       'Loan_Amount_Term', 'Credit_History', 'Property_Area', 'Loan_Status'],
      dtype='object')